from rai.search.searchenginemanager import *
from rai.web.web import *
from BeautifulSoup import BeautifulSoup
from rai.file.FileManager import *
from rai.formatter.formatter import *
import codecs
import re


manager = SearchEngineManager()
manager.setSearchQuery("tina turner person")
manager.setMaxResults(5)
manager.executeQuery()
results = manager.getResults()
print "DISPLAYING RESULTS"
index = 0
for res in results:
    #__dir__ = os.path.dirname(os.path.abspath(__file__))
    filename = "fetch" + str(index) + ".txt"
    #filepath = os.path.join(__dir__, filename)
    #print filepath
    #f = codecs.open('test', encoding='utf-8', mode='w')
    #f = open(filepath, 'w')
    
    rawfile = FileStore()
    rawfile.createFile(filename)
    
    url = res.Url
    summary = res.Summary
    title = res.Title
    rawfile.writeLine(url)
    rawfile.writeLine(summary)
    rawfile.writeLine(title)
    print "############################################"
    print title
    print summary
    print url
    print "--------------------------------------------"
    rawfile.writeLine("--------------------------------------------")
    
    urlFetcher = URLFetcher(url)
    rawHTML = urlFetcher.fetchRaw()
    rawfile.writeLine(rawHTML)
    rawfile.writeLine("--------------------------------------------")
    rawfile.closeFile()
    

    # remove not needed HTML tags (e.g. script, style)
    htmlCleaner = HTMLCleaner(rawHTML)
    clean_html = htmlCleaner.cleanHTML() # prettified html soup
    
    cleanHTMLFile = FileStore()
    filename = "fetch" + str(index) + ".txt.html"
    cleanHTMLFile.createFile(filename)
    cleanHTMLFile.writeLine(clean_html) # write clean html to file
    cleanHTMLFile.closeFile()

    # Gets pure text without tags, converts htmlentities    
    allPureText = htmlCleaner.getPureText() 
    
    textFile = FileStore()
    filename = "fetch" + str(index) + ".clean.txt"
    textFile.createFile(filename)
    for line in allPureText:
        try:
            if (len(line) > 0):
                textFile.writeCleanLine(line)
        except UnicodeEncodeError, e:
            #print "unicode error: " + unicode(line,'Latin-1' ,'replace')
            #print "unicode error: " + unicode(line,'Latin-1' ,'replace')
            #print line.encode('utf8')
            print decodedString.encode('utf8')
            
    textFile.closeFile()
    
    # create file with sentences, split on dot
    textFormatter = FormatText(allPureText)
    sentences = textFormatter.getSentences()
    
    sentencesFile = FileStore()
    filename = "fetch" + str(index) + ".sentences.txt"
    sentencesFile.createFile(filename)
    # write all sentences to file
    for sentence in sentences:
        sentencesFile.writeCleanLine(sentence)
    sentencesFile.closeFile()
    
    index = index + 1